Final Project
Florin Andrei
4/24/2021
rm(list = ls())
library(dplyr)
library(tidyr)
library(stringr)
library(varhandle)
library(nnet)
library(NeuralNetTools)
library(pROC)
library(gplots)
library(foreach)
library(parallel)
library(iterators)
library(doParallel)Dataset: https://www.kaggle.com/datasnaek/chess
games = read.csv("games.csv") %>% distinct(id, .keep_all = TRUE)
ds.size.orig = dim(games)
# Eliminate irrelevant variables:
# game ID, player ID
# rated game
# opening ECO code
# start and end time (bad data, insufficient precision)
# moves (I feel just analyzing strings here would not go very far)
games = games %>% select(-c("id", "rated", "white_id", "black_id", "opening_eco", "created_at", "last_move_at", "moves", "victory_status"))
# increment_code is two different things (together defining the "speed" of the game):
# - initial assigned time for the game (minutes)
# - time increment after each move (seconds)
# so let's split that column
times_temp = as.data.frame(str_split_fixed(games$increment_code, "\\+", 2))
games = cbind(games, times_temp) %>% rename(initial_time = V1) %>% rename(time_increment = V2) %>% select(-c("increment_code"))
games$initial_time = unfactor(games$initial_time)
games$time_increment = unfactor(games$time_increment)
# eliminate rare openings
# we only consider openings that appear in a minimum number of games
op.count.min = 100
ocount = as.vector(games %>% group_by(opening_name) %>% summarize(count = n()) %>% arrange(desc(count)))
ops.orig = dim(ocount)
ocount = ocount %>% filter(count >= op.count.min)
ops.kept = dim(ocount)
write.csv(ocount, file = "ocount.csv", row.names = F)
games = games %>% filter(opening_name %in% unfactor(ocount$opening_name))
ds.size.pruned = dim(games)
print(paste("size of original dataset:", ds.size.orig[1], "games."), quote = F)## [1] size of original dataset: 19113 games.print(paste("We only consider games with openings that occur more than", op.count.min, "times in the dataset."), quote = F)## [1] We only consider games with openings that occur more than 100 times in the dataset.print(paste("Total number of openings in the dataset:", ops.orig[1]), quote = F)## [1] Total number of openings in the dataset: 1477print(paste("Number of openings kept:", ops.kept[1]), quote = F)## [1] Number of openings kept: 39print(paste("Size of dataset after pruning of openings:", ds.size.pruned[1]), quote = F)## [1] Size of dataset after pruning of openings: 6506print(paste("After pruning, the dataset is", round(100 * ds.size.pruned[1] / ds.size.orig[1]), "% the size of the original."), quote = F)## [1] After pruning, the dataset is 34 % the size of the original.# remove draws. Two reasons:
# - we're aiming to win
# - multinomial regression is extremely slow at this size, so we only want 2 levels for the outcome
games = games %>% filter(winner != "draw") %>% droplevels()
print(paste("Size of dataset after removing draws:", dim(games)[1]), quote = F)## [1] Size of dataset after removing draws: 6220games$win.bin = rep(0, times = dim(games)[1])
games$win.bin[which(games$winner == "white")] = 1
games$win.bin = as.factor(games$win.bin)
games = games %>% select(-c("winner"))
summary(games)## turns white_rating black_rating
## Min. : 1.00 Min. : 793 Min. : 796
## 1st Qu.: 36.00 1st Qu.:1353 1st Qu.:1353
## Median : 55.00 Median :1509 Median :1504
## Mean : 59.03 Mean :1532 Mean :1528
## 3rd Qu.: 77.00 3rd Qu.:1703 3rd Qu.:1701
## Max. :222.00 Max. :2621 Max. :2516
##
## opening_name opening_ply
## Sicilian Defense : 334 Min. : 1.00
## Van't Kruijs Opening : 327 1st Qu.: 2.00
## Sicilian Defense: Bowdler Attack : 277 Median : 3.00
## French Defense: Knight Variation : 246 Mean : 3.38
## Scotch Game : 243 3rd Qu.: 4.00
## Scandinavian Defense: Mieses-Kotroc Variation: 241 Max. :11.00
## (Other) :4552
## initial_time time_increment win.bin
## Min. : 0.00 Min. : 0.000 0:3002
## 1st Qu.: 10.00 1st Qu.: 0.000 1:3218
## Median : 10.00 Median : 0.000
## Mean : 14.24 Mean : 5.023
## 3rd Qu.: 15.00 3rd Qu.: 7.000
## Max. :180.00 Max. :180.000
## par(mfrow = c(2, 2))
hist(games$turns, breaks = 100)
hist(games$white_rating, breaks = 100)
hist(games$black_rating, breaks = 100)
hist(log(games$opening_ply), breaks = 100)
The frequencies of openings we’ve kept:
head(ocount, n = 10)## # A tibble: 10 x 2
## opening_name count
## <fct> <int>
## 1 Sicilian Defense 349
## 2 Van't Kruijs Opening 342
## 3 Sicilian Defense: Bowdler Attack 290
## 4 French Defense: Knight Variation 260
## 5 Scotch Game 254
## 6 Scandinavian Defense: Mieses-Kotroc Variation 247
## 7 Queen's Pawn Game: Mason Attack 227
## 8 Queen's Pawn Game: Chigorin Variation 217
## 9 Scandinavian Defense 217
## 10 Horwitz Defense 208barplot(ocount$count, main = "Frequencies of openings")
Training / testing data split
This is a large dataset, even after all the pruning of data we’ve done so far. We can afford to just set aside some (dedicated) data for testing.
I will use a 75/25 split for training/testing.
Note: I was planning to use double-cross validation for logistic regression (the neural network would take a prohibitive amount of time). I ran out of time and, with only a few hours left to the deadline, I am going to skip double-CV. At least the size of the dataset is in the thousands, so the split should offer a decent test. Double-CV is on my practice list, in R and Python, for this summer, before the fall semester. :)
dsn = dim(games)[1]
t.ratio = 0.75
train.size = round(dsn * t.ratio)
test.start = train.size + 1
d.train = games[1:train.size, ]
d.test = games[test.start:dsn, ]
print("Training set size:", quote = F)## [1] Training set size:dim(d.train)[1]## [1] 4665print("Testing set size:", quote = F)## [1] Testing set size:dim(d.test)[1]## [1] 1555Plain logistic regression
Let’s train the regression model on the training slice of data:
set.seed(10)
f.log = glm(win.bin ~ ., data = d.train, family = "binomial")
summary(f.log)##
## Call:
## glm(formula = win.bin ~ ., family = "binomial", data = d.train)
##
## Deviance Residuals:
## Min 1Q Median 3Q Max
## -2.6469 -1.0423 0.3261 1.0298 2.8013
##
## Coefficients:
## Estimate Std. Error
## (Intercept) 0.8796688 0.3835904
## turns -0.0025493 0.0010131
## white_rating 0.0042032 0.0002009
## black_rating -0.0044359 0.0001942
## opening_nameCaro-Kann Defense -0.1966227 0.2994505
## opening_nameFour Knights Game: Italian Variation 0.2222888 0.4667178
## opening_nameFrench Defense #2 -0.4332493 0.3610534
## opening_nameFrench Defense: Exchange Variation 0.0959168 0.3903794
## opening_nameFrench Defense: Knight Variation -0.1888099 0.2823831
## opening_nameFrench Defense: Normal Variation 0.0448043 0.3361073
## opening_nameGiuoco Piano -0.5333089 0.4160872
## opening_nameHorwitz Defense -0.2423806 0.3024407
## opening_nameHungarian Opening -0.4992955 0.3623875
## opening_nameIndian Game -0.6026959 0.3231523
## opening_nameItalian Game 0.0855543 0.3883187
## opening_nameItalian Game: Anti-Fried Liver Defense -0.1420161 0.3835736
## opening_nameKing's Pawn Game: Leonardis Variation -0.4434333 0.3053497
## opening_nameKing's Pawn Game: Wayward Queen Attack -0.4646119 0.3039841
## opening_nameModern Defense -0.2406272 0.3167680
## opening_nameOwen Defense -0.6111593 0.3161092
## opening_namePhilidor Defense 0.0864527 0.3637600
## opening_namePhilidor Defense #2 -0.3829342 0.3100417
## opening_namePhilidor Defense #3 0.1762824 0.3400238
## opening_namePirc Defense #4 0.1637602 0.3706851
## opening_nameQueen's Gambit Accepted: Old Variation -0.2598652 0.3968254
## opening_nameQueen's Gambit Declined -0.0736039 0.3561222
## opening_nameQueen's Gambit Refused: Marshall Defense 0.3562995 0.3467057
## opening_nameQueen's Pawn 0.1728406 0.3545494
## opening_nameQueen's Pawn Game -0.6001289 0.3334554
## opening_nameQueen's Pawn Game #2 -0.2461893 0.3429364
## opening_nameQueen's Pawn Game: Chigorin Variation -0.1763997 0.2976340
## opening_nameQueen's Pawn Game: Mason Attack -0.2113953 0.2874968
## opening_nameQueen's Pawn Game: Zukertort Variation -0.3654132 0.3458986
## opening_nameRuy Lopez: Steinitz Defense 0.0726829 0.4001390
## opening_nameScandinavian Defense -0.7046092 0.2963105
## opening_nameScandinavian Defense: Mieses-Kotroc Variation 0.0063835 0.2921826
## opening_nameScotch Game -0.0896434 0.3659420
## opening_nameSicilian Defense -0.3958769 0.2718208
## opening_nameSicilian Defense: Bowdler Attack -0.4052731 0.2813373
## opening_nameSicilian Defense: Old Sicilian -0.6328364 0.3216639
## opening_nameSicilian Defense: Smith-Morra Gambit #2 -0.0002563 0.3321967
## opening_nameVan't Kruijs Opening -0.7158259 0.3192237
## opening_ply -0.0187628 0.0777944
## initial_time -0.0012262 0.0019967
## time_increment 0.0036287 0.0025812
## z value Pr(>|z|)
## (Intercept) 2.293 0.0218 *
## turns -2.516 0.0119 *
## white_rating 20.926 <2e-16 ***
## black_rating -22.847 <2e-16 ***
## opening_nameCaro-Kann Defense -0.657 0.5114
## opening_nameFour Knights Game: Italian Variation 0.476 0.6339
## opening_nameFrench Defense #2 -1.200 0.2302
## opening_nameFrench Defense: Exchange Variation 0.246 0.8059
## opening_nameFrench Defense: Knight Variation -0.669 0.5037
## opening_nameFrench Defense: Normal Variation 0.133 0.8940
## opening_nameGiuoco Piano -1.282 0.1999
## opening_nameHorwitz Defense -0.801 0.4229
## opening_nameHungarian Opening -1.378 0.1683
## opening_nameIndian Game -1.865 0.0622 .
## opening_nameItalian Game 0.220 0.8256
## opening_nameItalian Game: Anti-Fried Liver Defense -0.370 0.7112
## opening_nameKing's Pawn Game: Leonardis Variation -1.452 0.1464
## opening_nameKing's Pawn Game: Wayward Queen Attack -1.528 0.1264
## opening_nameModern Defense -0.760 0.4475
## opening_nameOwen Defense -1.933 0.0532 .
## opening_namePhilidor Defense 0.238 0.8121
## opening_namePhilidor Defense #2 -1.235 0.2168
## opening_namePhilidor Defense #3 0.518 0.6042
## opening_namePirc Defense #4 0.442 0.6587
## opening_nameQueen's Gambit Accepted: Old Variation -0.655 0.5126
## opening_nameQueen's Gambit Declined -0.207 0.8363
## opening_nameQueen's Gambit Refused: Marshall Defense 1.028 0.3041
## opening_nameQueen's Pawn 0.487 0.6259
## opening_nameQueen's Pawn Game -1.800 0.0719 .
## opening_nameQueen's Pawn Game #2 -0.718 0.4728
## opening_nameQueen's Pawn Game: Chigorin Variation -0.593 0.5534
## opening_nameQueen's Pawn Game: Mason Attack -0.735 0.4622
## opening_nameQueen's Pawn Game: Zukertort Variation -1.056 0.2908
## opening_nameRuy Lopez: Steinitz Defense 0.182 0.8559
## opening_nameScandinavian Defense -2.378 0.0174 *
## opening_nameScandinavian Defense: Mieses-Kotroc Variation 0.022 0.9826
## opening_nameScotch Game -0.245 0.8065
## opening_nameSicilian Defense -1.456 0.1453
## opening_nameSicilian Defense: Bowdler Attack -1.441 0.1497
## opening_nameSicilian Defense: Old Sicilian -1.967 0.0491 *
## opening_nameSicilian Defense: Smith-Morra Gambit #2 -0.001 0.9994
## opening_nameVan't Kruijs Opening -2.242 0.0249 *
## opening_ply -0.241 0.8094
## initial_time -0.614 0.5391
## time_increment 1.406 0.1598
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## (Dispersion parameter for binomial family taken to be 1)
##
## Null deviance: 6462.4 on 4664 degrees of freedom
## Residual deviance: 5500.0 on 4620 degrees of freedom
## AIC: 5590
##
## Number of Fisher Scoring iterations: 4Neural network
performance: the ratio of correct predictions / total predictions
We need to choose the best size and decay for the network, that will maximize performance.
The performance matrix, defined below, holds performance ratios at different decay values (rows) and different network sizes (columns).
The random seed affects performance greatly. Testing performance at many different seeds, then averaging the result, should provide a better picture. The code below does exactly that: it computes performance matrices using various seeds, then calculates the average matrix. The average is then used for choosing the best size and decay.
decay.list = seq(0.5, 5, by = 0.05)
size.list = seq(1, 16, by = 1)
seed.list = 1000:1047The following block takes half a day to run on a reasonably new gaming PC, even with doParallel, so it has been disabled here. The end result of it is the perf-avg.csv file, which is included in the submitted files, and will be read in the next block, to make sure the whole notebook runs fine start-to-finish.
cl <- parallel::makeCluster(detectCores())
doParallel::registerDoParallel(cl)
for (tseed in seed.list) {
# performance = ratio of correct predictions to total predictions
perf = matrix(, nr = length(decay.list), nc = length(size.list))
print(tseed)
print(size.list)
for (sz in size.list) {
cat(" ", sz)
perf.col = matrix(, nr = length(decay.list), nc = 1)
perf.col <- foreach (dc = decay.list, .combine = rbind) %dopar% {
set.seed(tseed)
f.nn = nnet::nnet(win.bin ~ ., data = d.train, size = sz, MaxNWts = 100000, maxit = 10000, decay = dc, trace = F)
stopifnot(f.nn$convergence == 0)
tp.nn = table(factor(predict(f.nn, d.test, type = "class"), levels = c(0, 1)), d.test$win.bin)
perf.col[which(decay.list == dc), 1] = (tp.nn[1, 1] + tp.nn[2, 2]) / sum(tp.nn)
}
perf[, which(size.list == sz)] = perf.col
}
print("", quote = F)
pfname = paste0("perf", tseed, ".csv")
write.csv(perf, file = pfname)
}
parallel::stopCluster(cl)
perf = matrix(0, nr = length(decay.list), nc = length(size.list))
for (sd in seed.list) {
pmf = paste0("perf", sd, ".csv")
pm = as.matrix(read.csv(pmf))[, -1]
phmf = paste0("frame-perf", sd, ".png")
png(filename = phmf, width = 1600, height = 900)
heatmap.2(exp(exp(exp(pm))) ^ 2, dendrogram = "none", Rowv = F, Colv = F, main = paste0("exp(exp(exp(perf))) ^ 2 at seed = ", sd))
dev.off()
perf = perf + pm
}
perf = perf / length(seed.list)
# animated loop showing performance at different seeds
# requires ImageMagick
shell("convert -delay 20 -loop 0 frame-perf*.png perf-anim.gif")
write.csv(perf, file = "perf-avg.csv", row.names = F)Histogram of performance values from the averaged matrix generated above:
perf = as.matrix(read.csv("perf-avg.csv"))
hist(perf, breaks = 100, col = "lightblue", main = "Histogram of performance")
Performance heat map:
The horizontal axis shows sizes, the vertical axis shows decays. Each frame is a different seed value. Each cell contains the performance of the model: yellow is good, red is bad.
performance heat maps at different random seeds
Average performance heatmap over several dozen seed values - this is the average generated over all seed values:
heatmap.2(exp(exp(exp(perf))) ^ 2, dendrogram = "none", Rowv = F, Colv = F, main = "exp(exp(exp(perf))) ^ 2")
max.ind = which(perf == max(perf), arr.ind = T)
decay.best = max.ind[1]
size.best = max.ind[2]
print(paste("Best performance:", max(perf)), quote = F)## [1] Best performance: 0.67591103965702print(paste("Best decay:", decay.list[decay.best], "out of", min(decay.list), "to", max(decay.list)), quote = F)## [1] Best decay: 3.55 out of 0.5 to 5print(paste("Best size:", size.list[size.best], "out of", min(size.list), "to", max(size.list)), quote = F)## [1] Best size: 2 out of 1 to 16Averaging over seed values provided the best performance with a fairly parsimonious model (small size, high decay), so we’re going to keep these “best” values for size and decay.
Evaluate the model performance
Using the best size and the best decay, train the network on the training data slice. The logistic regression model is already trained.
set.seed(10)
f.nn = nnet(win.bin ~ ., data = d.train, size = size.list[size.best], MaxNWts = 100000, maxit = 10000, decay = decay.list[decay.best])## # weights: 93
## initial value 3303.669960
## iter 10 value 3230.200256
## iter 20 value 3118.820454
## iter 30 value 2976.376020
## iter 40 value 2959.650678
## iter 50 value 2920.478791
## iter 60 value 2894.689497
## iter 70 value 2842.576596
## iter 80 value 2819.628814
## iter 90 value 2804.450625
## iter 100 value 2800.297132
## iter 110 value 2793.988489
## iter 120 value 2790.644013
## iter 130 value 2790.603701
## iter 140 value 2790.598186
## final value 2790.598041
## convergedUsing both models, make predictions on the testing slice of the dataset, and compare with reality.
tp.log = table(predict(f.log, d.test, type = "response") > 0.5, d.test$win.bin)
tp.nn = table(predict(f.nn, d.test, type = "class"), d.test$win.bin)
tp.log##
## 0 1
## FALSE 476 235
## TRUE 267 577tp.nn##
## 0 1
## 0 474 230
## 1 269 582perf.log = (tp.log[1, 1] + tp.log[2, 2]) / sum(tp.log)
perf.nn = (tp.nn[1, 1] + tp.nn[2, 2]) / sum(tp.nn)
print("", quote = F)## [1]print(paste("Logistic regression performance:", perf.log), quote = F)## [1] Logistic regression performance: 0.677170418006431print(paste("Neural network performance: ", perf.nn), quote = F)## [1] Neural network performance: 0.679099678456592ROC curves:
par(pty = 's')
plot(roc(predictor = predict(f.log, d.test, type = "response"), response = d.test$win.bin), col = "blue")
par(new = T)
plot(roc(predictor = predict(f.nn, d.test, type = "raw"), response = d.test$win.bin), col = "red")
legend("bottomright", legend = c("logistic regression", "neural network"), lty = c(1, 1), col = c("blue", "red"))
I was hoping for better performance, but it is what it is. The neural network performs slightly better.
We’re going to pick the neural network to make predictions.
Predictions
Train the neural network on all existing data:
model.nn = nnet(win.bin ~ ., data = games, size = size.list[size.best], MaxNWts = 100000, maxit = 10000, decay = decay.list[decay.best])## # weights: 93
## initial value 4357.327920
## iter 10 value 3894.744310
## iter 20 value 3843.542165
## iter 30 value 3833.064106
## iter 40 value 3815.050081
## iter 50 value 3777.797957
## iter 60 value 3741.240091
## iter 70 value 3733.209817
## iter 80 value 3730.583319
## iter 90 value 3730.040393
## iter 100 value 3729.919077
## iter 110 value 3729.871542
## iter 120 value 3729.788072
## iter 130 value 3729.758219
## final value 3729.757389
## convergedThe home team player has an ELO of 1500. They are going to play two games:
- play as white against an opponent at ELO = 1600
- play as black against an opponent at ELO = 1400
Which openings will maximize the chances of winning?
We will assume mean values for the other predictors (time, number of turns).
We will create one dataframe for each game; the only thing that changes from one row to another within each dataframe is the opening. Then we will use the model to see which opening is predicted to maximize the chances of winning.
elo.home = 1500
elo.p1 = 1600
elo.p2 = 1400
init.time.mean = exp(mean(log(games$initial_time), trim = 0.1))
time.incr.mean = exp(mean(log(games$time_increment[which(games$time_increment > 0)])))
o.ply.mean = round(exp(mean(log(games$opening_ply[which(games$opening_ply > 0)]))))
turns.mean = round(mean(games$turns))
op.tot = length(ocount$opening_name)
games.white = data.frame(
turns = rep(turns.mean, times = op.tot),
white_rating = rep(elo.home, times = op.tot),
black_rating = rep(elo.p1, times = op.tot),
opening_name = ocount$opening_name,
opening_ply = rep(o.ply.mean, times = op.tot),
initial_time = rep(init.time.mean, times = op.tot),
time_increment = rep(time.incr.mean, times = op.tot),
win.bin = as.factor(rep(1, times = op.tot))
)
games.black = data.frame(
turns = rep(turns.mean, times = op.tot),
white_rating = rep(elo.p2, times = op.tot),
black_rating = rep(elo.home, times = op.tot),
opening_name = ocount$opening_name,
opening_ply = rep(o.ply.mean, times = op.tot),
initial_time = rep(init.time.mean, times = op.tot),
time_increment = rep(time.incr.mean, times = op.tot),
win.bin = as.factor(rep(0, times = op.tot))
)
summary(games.white)## turns white_rating black_rating
## Min. :59 Min. :1500 Min. :1600
## 1st Qu.:59 1st Qu.:1500 1st Qu.:1600
## Median :59 Median :1500 Median :1600
## Mean :59 Mean :1500 Mean :1600
## 3rd Qu.:59 3rd Qu.:1500 3rd Qu.:1600
## Max. :59 Max. :1500 Max. :1600
##
## opening_name opening_ply initial_time
## Bishop's Opening : 1 Min. :3 Min. :10.75
## Caro-Kann Defense : 1 1st Qu.:3 1st Qu.:10.75
## Four Knights Game: Italian Variation: 1 Median :3 Median :10.75
## French Defense #2 : 1 Mean :3 Mean :10.75
## French Defense: Exchange Variation : 1 3rd Qu.:3 3rd Qu.:10.75
## French Defense: Knight Variation : 1 Max. :3 Max. :10.75
## (Other) :33
## time_increment win.bin
## Min. :7.386 1:39
## 1st Qu.:7.386
## Median :7.386
## Mean :7.386
## 3rd Qu.:7.386
## Max. :7.386
## summary(games.black)## turns white_rating black_rating
## Min. :59 Min. :1400 Min. :1500
## 1st Qu.:59 1st Qu.:1400 1st Qu.:1500
## Median :59 Median :1400 Median :1500
## Mean :59 Mean :1400 Mean :1500
## 3rd Qu.:59 3rd Qu.:1400 3rd Qu.:1500
## Max. :59 Max. :1400 Max. :1500
##
## opening_name opening_ply initial_time
## Bishop's Opening : 1 Min. :3 Min. :10.75
## Caro-Kann Defense : 1 1st Qu.:3 1st Qu.:10.75
## Four Knights Game: Italian Variation: 1 Median :3 Median :10.75
## French Defense #2 : 1 Mean :3 Mean :10.75
## French Defense: Exchange Variation : 1 3rd Qu.:3 3rd Qu.:10.75
## French Defense: Knight Variation : 1 Max. :3 Max. :10.75
## (Other) :33
## time_increment win.bin
## Min. :7.386 0:39
## 1st Qu.:7.386
## Median :7.386
## Mean :7.386
## 3rd Qu.:7.386
## Max. :7.386
## Predict the outcome of the game played as white against the stronger opponent. Here, prediction = 1 means highest odds of white winning, so we will sort the winning in decreasing order.
p.white = predict(model.nn, games.white, type = "raw")
pred.white = data.frame(
opening = games.white$opening_name,
win_prob = as.vector(p.white)
)
best.white = pred.white[order(pred.white$win_prob, decreasing = T),]
write.csv(best.white, file = "best_white.csv", row.names = F)
best.white## opening win_prob
## 25 Queen's Pawn 0.4501371
## 6 Scandinavian Defense: Mieses-Kotroc Variation 0.4377557
## 38 Pirc Defense #4 0.4295530
## 34 Sicilian Defense: Smith-Morra Gambit #2 0.4198934
## 37 Italian Game 0.4196096
## 26 Queen's Gambit Refused: Marshall Defense 0.4164996
## 11 Caro-Kann Defense 0.4140233
## 4 French Defense: Knight Variation 0.4138846
## 30 Queen's Gambit Declined 0.4131132
## 23 Philidor Defense 0.4128466
## 12 Philidor Defense #3 0.4107184
## 28 French Defense: Normal Variation 0.3987685
## 31 Queen's Pawn Game: Zukertort Variation 0.3956471
## 18 Four Knights Game: Italian Variation 0.3919486
## 36 French Defense: Exchange Variation 0.3915099
## 32 Queen's Pawn Game #2 0.3887330
## 10 Horwitz Defense 0.3870948
## 27 Bishop's Opening 0.3768875
## 8 Queen's Pawn Game: Chigorin Variation 0.3720320
## 13 Philidor Defense #2 0.3713351
## 33 Queen's Gambit Accepted: Old Variation 0.3705794
## 21 King's Pawn Game: Leonardis Variation 0.3703631
## 24 Ruy Lopez: Steinitz Defense 0.3631123
## 22 Queen's Pawn Game 0.3624154
## 15 Modern Defense 0.3617383
## 7 Queen's Pawn Game: Mason Attack 0.3616814
## 35 French Defense #2 0.3595915
## 29 Hungarian Opening 0.3589511
## 16 Italian Game: Anti-Fried Liver Defense 0.3533848
## 14 Indian Game 0.3499980
## 5 Scotch Game 0.3493635
## 3 Sicilian Defense: Bowdler Attack 0.3423866
## 17 King's Pawn Game: Wayward Queen Attack 0.3423066
## 19 Owen Defense 0.3383961
## 9 Scandinavian Defense 0.3357196
## 1 Sicilian Defense 0.3332519
## 20 Sicilian Defense: Old Sicilian 0.3257028
## 39 Giuoco Piano 0.3184512
## 2 Van't Kruijs Opening 0.3121804Predict the outcome of the game played as black against the weaker opponent. Here prediction = 0 means the highest odds of black winning, so we use sort in increasing order of winning.
p.black = predict(model.nn, games.black, type = "raw")
pred.black = data.frame(
opening = games.black$opening_name,
win_prob = as.vector(p.black)
)
best.black = pred.black[order(pred.black$win_prob, decreasing = F),]
write.csv(best.black, file = "best_black.csv", row.names = F)
best.black## opening win_prob
## 2 Van't Kruijs Opening 0.3130230
## 39 Giuoco Piano 0.3193277
## 20 Sicilian Defense: Old Sicilian 0.3266225
## 1 Sicilian Defense 0.3342153
## 9 Scandinavian Defense 0.3366948
## 19 Owen Defense 0.3393778
## 17 King's Pawn Game: Wayward Queen Attack 0.3432906
## 3 Sicilian Defense: Bowdler Attack 0.3433769
## 5 Scotch Game 0.3503893
## 14 Indian Game 0.3510118
## 16 Italian Game: Anti-Fried Liver Defense 0.3544171
## 29 Hungarian Opening 0.3600075
## 35 French Defense #2 0.3606629
## 7 Queen's Pawn Game: Mason Attack 0.3627448
## 15 Modern Defense 0.3628062
## 22 Queen's Pawn Game 0.3634939
## 24 Ruy Lopez: Steinitz Defense 0.3641951
## 21 King's Pawn Game: Leonardis Variation 0.3714704
## 33 Queen's Gambit Accepted: Old Variation 0.3716748
## 13 Philidor Defense #2 0.3724496
## 8 Queen's Pawn Game: Chigorin Variation 0.3731463
## 27 Bishop's Opening 0.3780086
## 10 Horwitz Defense 0.3882349
## 32 Queen's Pawn Game #2 0.3898873
## 36 French Defense: Exchange Variation 0.3926739
## 18 Four Knights Game: Italian Variation 0.3931091
## 31 Queen's Pawn Game: Zukertort Variation 0.3968183
## 28 French Defense: Normal Variation 0.3999505
## 12 Philidor Defense #3 0.4119202
## 23 Philidor Defense 0.4140498
## 30 Queen's Gambit Declined 0.4143256
## 4 French Defense: Knight Variation 0.4151011
## 11 Caro-Kann Defense 0.4152337
## 26 Queen's Gambit Refused: Marshall Defense 0.4177115
## 37 Italian Game 0.4208370
## 34 Sicilian Defense: Smith-Morra Gambit #2 0.4211184
## 38 Pirc Defense #4 0.4307895
## 6 Scandinavian Defense: Mieses-Kotroc Variation 0.4389943
## 25 Queen's Pawn 0.4513830TODO
I am not sure whether the model’s predictions are stable when the other variables change. E.g., would it be better to aim for a longer game, or shorter?